package org.example

import com.hankcs.hanlp.HanLP
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary
import com.hankcs.hanlp.summary.TextRankKeyword
import com.hankcs.hanlp.tokenizer.StandardTokenizer
import org.apache.hadoop.shaded.org.apache.commons.lang3.builder.StandardToStringStyle
import org.apache.spark.sql.SparkSession

import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`

object words39 {
  def main(args: Array[String]): Unit = {
    //SPARK运行环境
    val spark = SparkSession
      .builder()
      .master("local[*]")
      .appName("spark")
      .getOrCreate()

    val sc = spark.sparkContext

    sc.textFile("E:\\23spark209\\fqy09\\src\\main\\resources\\word.txt")
      .flatMap(_.split(" "))
      .map(x => (x, 1))
      .reduceByKey((x,y) => x + y)
      .sortBy(_._2,true)
      .foreach(println)

    // 中文分词
    val chinese1 = HanLP.segment("张三上午没吃早餐，中午想吃牛肉面")
    //  println(chinese1)
    //  println(chinese1.asScala.map(_.word.trim))


    val chinese2 = StandardTokenizer.segment("明天++清明节++过两周++五一")
    //  println(chinese2)
    //  println(chinese2.asScala.map(_.word.replaceAll("\\s","")))

    TextRankKeyword.getKeywordList("看！广东这几个地方拆迁！征收位置、面积、补偿和安置标准都已公布，快来看看有你家吗",5)
    //    .forEach(println)
    val chinese3 = """11:20:00 23510206030209 [spark大数据] www.baidu.com""".split("\\s+")
    println(chinese3(2).replaceAll("\\[|\\]", ""))
    val textArr = Array(
      "su7爆燃遇难者母亲清空相关微博",
      "甲亢哥被重庆甜妹教育了",
      "胡歌老婆是他前助理",
      "金价再起飞",
      "你能接受与伴侣的年龄差是多少"
    )
    val textRDD = sc.parallelize(textArr)
    val textResult = textRDD.map {
      text =>
        val keyword = TextRankKeyword.getKeywordList(text, 5).toString
        val words = transform(text)
        (text, keyword, words)
    } // RDD[(String, String, List[String])]
    textResult.take(1).foreach(println)
  }

  // 结果转换，可以不显示词性
  def transform(sentense: String): List[String] = {
    val list = StandardTokenizer.segment(sentense)
    CoreStopWordDictionary.apply(list)
    list.map(x => x.word.replaceAll(" ", "")).toList

    //  sc.stop()

  }

}
